{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4226f6f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "from dotenv import load_dotenv\n",
    "from openai import OpenAI\n",
    "from IPython.display import Markdown, display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4cdb4a69",
   "metadata": {},
   "outputs": [],
   "source": [
    "load_dotenv(override=True)\n",
    "\n",
    "openai_api_key = os.getenv(\"OPENAI_API_KEY\")\n",
    "google_api_key = os.getenv(\"GOOGLE_API_KEY\")\n",
    "\n",
    "if openai_api_key is None:\n",
    "    raise ValueError(\"OPENAI_API_KEY is not set\")\n",
    "\n",
    "if google_api_key is None:\n",
    "    raise ValueError(\"GOOGLE_API_KEY is not set\")\n",
    "\n",
    "\n",
    "\n",
    "# The API we know well\n",
    "# I've updated this with the latest model, but it can take some time because it likes to think!\n",
    "# Replace the model with gpt-4.1-mini if you'd prefer not to wait 1-2 mins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "31c74663",
   "metadata": {},
   "outputs": [],
   "source": [
    "request = \"Please come up with a challenging, nuanced question that I can ask a number of LLMs to generate a code for algorithm like binary tree for live coding competition. \"\n",
    "request += \"Answer only with the question, no explanation.\"\n",
    "messages = [{\"role\": \"user\", \"content\": request}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0b9dc1d7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'role': 'user', 'content': 'Please come up with a challenging, nuanced question that I can ask a number of LLMs to generate a code for algorithm like binary tree for live coding competition. Answer only with the question, no explanation.'}]\n"
     ]
    }
   ],
   "source": [
    "print(messages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "298de8ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "How would you implement a binary tree in Python that includes methods for insertion, deletion, traversal (in-order, pre-order, post-order), and searching for a specific value, while also ensuring balanced height after each insertion?\n"
     ]
    }
   ],
   "source": [
    "openai = OpenAI()\n",
    "response = openai.chat.completions.create(\n",
    "    model=\"gpt-4o-mini\",\n",
    "    messages=messages,\n",
    ")\n",
    "question = response.choices[0].message.content\n",
    "print(question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b26c539a",
   "metadata": {},
   "outputs": [],
   "source": [
    "competitors = []\n",
    "answers = []\n",
    "messages = [{\"role\": \"user\", \"content\": question}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cdd1c225",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"gpt-5-mini\"\n",
    "\n",
    "openai = OpenAI()\n",
    "response = openai.chat.completions.create(\n",
    "    model=\"gpt-5-mini\",\n",
    "    messages=messages,\n",
    ")\n",
    "answer = response.choices[0].message.content\n",
    "\n",
    "display(Markdown(answer))\n",
    "answers.append(answer)\n",
    "competitors.append(model_name)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad9ccdb4",
   "metadata": {},
   "outputs": [],
   "source": [
    "gemini = OpenAI(api_key=google_api_key, base_url=\"https://generativelanguage.googleapis.com/v1beta/openai/\")\n",
    "model_name = \"gemini-2.5-flash\"\n",
    "\n",
    "response = gemini.chat.completions.create(model=model_name, messages=messages)\n",
    "answer = response.choices[0].message.content\n",
    "\n",
    "display(Markdown(answer))\n",
    "competitors.append(model_name)\n",
    "answers.append(answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14709041",
   "metadata": {},
   "outputs": [],
   "source": [
    "ollama = OpenAI(base_url=\"http://localhost:11434/v1\")\n",
    "model_name = \"phi3:latest\"\n",
    "\n",
    "response = ollama.chat.completions.create(model=model_name, messages=messages)\n",
    "answer = response.choices[0].message.content\n",
    "\n",
    "display(Markdown(answer))\n",
    "competitors.append(model_name)\n",
    "answers.append(answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd5e23f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(competitors)\n",
    "print(answers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96a5c917",
   "metadata": {},
   "outputs": [],
   "source": [
    "# It's nice to know how to use \"zip\"\n",
    "for competitor, answer in zip(competitors, answers):\n",
    "    print(f\"Competitor: {competitor}\\n\\n{answer}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "4e71c1c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's bring this together - note the use of \"enumerate\"\n",
    "\n",
    "together = \"\"\n",
    "for index, answer in enumerate(answers):\n",
    "    together += f\"# Response from competitor {index+1}\\n\\n\"\n",
    "    together += answer + \"\\n\\n\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db4b67c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(together)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "dbf92ba2",
   "metadata": {},
   "outputs": [],
   "source": [
    "judge = f\"\"\"You are judging a competition between {len(competitors)} competitors.\n",
    "Each model has been given this question:\n",
    "\n",
    "{question}\n",
    "\n",
    "Your job is to evaluate each response for clarity and strength of argument, and rank them in order of best to worst.\n",
    "Respond with JSON, and only JSON, with the following format:\n",
    "{{\"results\": [\"best competitor number\", \"second best competitor number\", \"third best competitor number\", ...]}}\n",
    "\n",
    "Here are the responses from each competitor:\n",
    "\n",
    "{together}\n",
    "\n",
    "Now respond with the JSON with the ranked order of the competitors, nothing else. Do not include markdown formatting or code blocks.\"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3eebf961",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(judge)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "5953feb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "judge_messages = [{\"role\": \"user\", \"content\": judge}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8bde0152",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Judgement time!\n",
    "\n",
    "openai = OpenAI()\n",
    "response = openai.chat.completions.create(\n",
    "    model=\"gpt-5-mini\",\n",
    "    messages=judge_messages,\n",
    ")\n",
    "results = response.choices[0].message.content\n",
    "print(results)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c8f1410",
   "metadata": {},
   "outputs": [],
   "source": [
    "# OK let's turn this into results!\n",
    "\n",
    "results_dict = json.loads(results)\n",
    "ranks = results_dict[\"results\"]\n",
    "for index, result in enumerate(ranks):\n",
    "    competitor = competitors[int(result)-1]\n",
    "    print(f\"Rank {index+1}: {competitor}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5e6f540",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
